package com.ruoyi.crm.crawling.util;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.ruoyi.common.annotation.Log;
import com.ruoyi.common.enums.BusinessType;
import com.ruoyi.crm.crawling.domain.CrmCrawling;
import com.ruoyi.crm.crawling.service.ICrmCrawlingService;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Links;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.rocks.BreadthCrawler;

/**
 * Crawling news from github news 自动爬取新闻网站，继承 BreadthCrawler（广度爬虫）
 * BreadthCrawler 是 WebCollector 最常用的爬取器之一
 *
 * @author hu
 */
public class DemoAutoNewsCrawler extends BreadthCrawler {
	
    private ICrmCrawlingService crmCrawlingService;
	
	/**
	 * @param crawlPath crawlPath is the path of the directory which maintains
	 *                  information of this crawler
	 * @param autoParse if autoParse is true,BreadthCrawler will auto extract links
	 *                  which match regex rules from pag
	 */
	public DemoAutoNewsCrawler(String crawlPath, boolean autoParse, ICrmCrawlingService crmCrawlingService) {
		super(crawlPath, autoParse);

		this.crmCrawlingService = crmCrawlingService;
		
		SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd");
		Calendar calendar = new GregorianCalendar();
		calendar.setTime(new Date());
		calendar.add(Calendar.DATE,-1);
		String lastDate= sdf.format(calendar.getTime());
		
		/**
		 * 循环添加了3类种子+分页
		 */
//		for (int pageIndex = 2; pageIndex <= 2; pageIndex++) {
//			String seedUrl = String.format("http://www.ccgp-shandong.gov.cn/sdgp2017/site/listnew.jsp?grade=&colcode=2501&curpage=%d", pageIndex);
			this.addSeed("http://www.ccgp-shandong.gov.cn/sdgp2017/site/listnew.jsp?grade=&colcode=2501&pdate=" + lastDate);
//		}
//		for (int pageIndex = 1; pageIndex <= 1; pageIndex++) {
//			String seedUrl = String.format("http://www.ccgp-shandong.gov.cn/sdgp2017/site/listnew.jsp?grade=&colcode=2500&curpage=%d", pageIndex);
			this.addSeed("http://www.ccgp-shandong.gov.cn/sdgp2017/site/listnew.jsp?grade=&colcode=2500&pdate=" + lastDate);
//		}
//		for (int pageIndex = 1; pageIndex <= 1; pageIndex++) {
//			String seedUrl = String.format("http://www.ccgp-shandong.gov.cn/sdgp2017/site/listnew.jsp?grade=&colcode=2504&curpage=%d", pageIndex);
			this.addSeed("http://www.ccgp-shandong.gov.cn/sdgp2017/site/listnew.jsp?grade=&colcode=2504&pdate=" + lastDate);
//		}
		

		/** 设置线程数 */
		setThreads(50);
		getConf().setTopN(0);
		getConf().setExecuteInterval(1000);//设置每个线程的爬取间隔
		getConf().setReadTimeout(1000);

		/**
		 * 是否进行断电爬取，默认为 false setResumable(true);
		 */
	}

	/**
	 * 必须重写 visit 方法，作用是: 在整个抓取过程中,只要抓到符合要求的页面,webCollector 就会回调该方法,并传入一个包含了页面所有信息的
	 * page 对象
	 *
	 * @param page
	 * @param next
	 */
	@Override
	public void visit(Page page, CrawlDatums next) {
		String url = page.url();
//		System.out.println("================================================================" + url);
		// 如果遇到301或者302，手动跳转（将任务加到next中）
	    // 并且复制任务的meta
	    if(page.code() == 301 || page.code() == 302){
	        next.add(page.location());
	        return;
	    }
		
		if (url.indexOf("http://www.ccgp-shandong.gov.cn/sdgp2017/site/listcontXxgk.jsp?colcode=2501&id=") > -1) {
			try {
				//省需求数据拼装入库
				assemblyDataxq(page, "2501");
			} catch (Exception e) {
				e.printStackTrace();
			}
			
 
        }else if(url.indexOf("http://www.ccgp-shandong.gov.cn/sdgp2017/site/listcontnew.jsp?colcode=2500&id=") > -1){
        	try {
				//省意向数据拼装入库
				assemblyDatayx(page, "2500");
			} catch (Exception e) {
				e.printStackTrace();
			}
        }else if(url.indexOf("http://www.ccgp-shandong.gov.cn/sdgp2017/site/listcontnew.jsp?colcode=2504&id=") > -1){
        	try {
				//市县数据拼装入库
				assemblyDatayx(page, "2504");
			} catch (Exception e) {
				e.printStackTrace();
			}
        }else {
        	Elements eleList = page.doc().getElementsByClass("title");
    		Links links = new Links();
    		for(int i = 0; i < eleList.size(); i++) {
    			links.add("http://www.ccgp-shandong.gov.cn" + eleList.get(i).getElementsByTag("a").attr("href"));
    		}
    		int totalnum = Integer.valueOf(page.doc().getElementById("totalnum").text().trim());//总页数
    		int curpage = Integer.valueOf(page.doc().getElementById("curpage").text().trim());//当前页数
    		if(curpage == 1) {
    			for(int i = 2; i <= totalnum; i++) {
        			links.add(url + "&curpage=" + i);
        		}
    		}
    		
    		next.add(links);
        }
		
	}
	
	/**
	 * 生需求数据拼装+数据入库-意向
	 * @param page
	 * @throws ParseException 
	 */
	@Log(title = "数据处理-意向", businessType = BusinessType.INSERT)
	public void assemblyDatayx(Page page, String type) throws ParseException {
		
		CrmCrawling crawling = new CrmCrawling();
		
		//原文地址
		crawling.setUrl(page.url());
		//类型
		crawling.setType(type);
		
		SimpleDateFormat formatter=new SimpleDateFormat("yyyy年MM月dd日HH时mm分");
		SimpleDateFormat formatter1=new SimpleDateFormat("yyyy年MM月");
		SimpleDateFormat formatter2=new SimpleDateFormat("yyyy年MM月dd日");
		
		Document doc = page.doc();
		
		Elements contents = doc.getElementsByClass("content");//文章主体
		crawling.setDemandContent(contents.html());
		Elements ps = null;//正文内容
		Elements conTable = null;
		if(contents.size() > 0) {
			ps = contents.get(0).getElementsByTag("table");
			//获取发布时间、发布人
			Elements fbsj_fbr = doc.getElementsByTag("midea");
			if(fbsj_fbr.size() > 1) {
//					System.out.println("发布时间=" + fbsj_fbr.get(0).text().replace("发布时间：", ""));
				crawling.setPublishTime(formatter.parse(fbsj_fbr.get(0).text().replace("发布时间：", "")));
//					System.out.println("发布人=" + fbsj_fbr.get(1).text().replace("发布人：", ""));
				crawling.setPublisher(fbsj_fbr.get(1).text().replace("发布人：", ""));
			}
			
			if(ps.get(0).getElementsByTag("div").size() == 0) {//市县需求
				
				Elements trs = ps.get(0).getElementsByTag("tr").get(0).getElementsByTag("table").get(0).getElementsByTag("tr");
				
				//获取项目名称
				if(null != trs && trs.size() > 0) {
					crawling.setDemandName(trs.get(0).text());//项目名称
				}
				
				
				
				//获取概述、预算
				if(null != trs && trs.size() > 1) {
//					System.out.println("概述、预算=" + ps.get(1).text());
					crawling.setSummary(trs.get(1).text());
				}
				
				//获取公示时间
				if(null != trs && trs.size() > 4) {
					System.out.println("开始时间=" + trs.get(4).text().substring(trs.get(4).text().lastIndexOf("：自"), trs.get(4).text().lastIndexOf("起，")).replace("：自", ""));
					crawling.setStarttime(formatter2.parse(trs.get(4).text().substring(trs.get(4).text().lastIndexOf("：自"), trs.get(4).text().lastIndexOf("起，")).replace("：自", "")));
//					System.out.println("结束时间=" + ps.get(6).text().substring(ps.get(6).text().lastIndexOf("，至"), ps.get(6).text().lastIndexOf("止")).replace("，至", ""));
					crawling.setEndtime(formatter2.parse(trs.get(4).text().substring(trs.get(4).text().lastIndexOf("，至"), trs.get(4).text().lastIndexOf("止")).replace("，至", "")));
				}
				
				//获取联系方式（采购单位、代理单位）
				if(null != ps && trs.size() > 7) {
//					System.out.println("采购单位=" + ps.get(10).text().substring(ps.get(10).text().lastIndexOf("采购单位："), ps.get(10).text().lastIndexOf("地址：")).replace("采购单位：", ""));
					crawling.setBuyingUnit(trs.get(7).text().substring(trs.get(7).text().lastIndexOf("采购单位："), trs.get(7).text().lastIndexOf("地址：")).replace("采购单位：", ""));
//					System.out.println("地址=" + ps.get(10).text().substring(ps.get(10).text().lastIndexOf("地址：")));
					crawling.setAddress(trs.get(7).text().substring(trs.get(7).text().lastIndexOf("地址：")).replace("地址：", ""));
				}
				if(null != ps && trs.size() > 8) {
//					System.out.println("联系人=" + ps.get(11).text().substring(ps.get(11).text().lastIndexOf("联系人："), ps.get(11).text().lastIndexOf("联系方式")).replace("联系人：", ""));
					crawling.setContacts(trs.get(8).text().substring(trs.get(8).text().lastIndexOf("联系人："), trs.get(8).text().lastIndexOf("联系方式")).replace("联系人：", ""));
//					System.out.println("联系方式=" + ps.get(11).text().substring(ps.get(11).text().lastIndexOf("联系方式：")));
					crawling.setContactsTel(trs.get(8).text().substring(trs.get(8).text().lastIndexOf("联系方式：")).replace("联系方式：", ""));
				}
				if(null != ps && trs.size() > 9) {
//					System.out.println("采购代理机构=" + ps.get(12).text().substring(ps.get(12).text().lastIndexOf("采购代理机构："), ps.get(12).text().lastIndexOf("地址")).replace("采购代理机构：", ""));
					crawling.setAgency(trs.get(9).text().substring(trs.get(9).text().lastIndexOf("采购代理机构："), trs.get(9).text().lastIndexOf("地址")).replace("采购代理机构：", ""));
//					System.out.println("地址=" + ps.get(12).text().substring(ps.get(12).text().lastIndexOf("地址：")));
					crawling.setAgencyAddress(trs.get(9).text().substring(trs.get(9).text().lastIndexOf("地址：")).replace("地址：", ""));
				}
				if(null != ps && trs.size() > 10) {
//					System.out.println("联系人=" + ps.get(13).text().substring(ps.get(13).text().lastIndexOf("联系人："), ps.get(13).text().lastIndexOf("联系方式")).replace("联系人：", ""));
					crawling.setAgencyContacts(trs.get(10).text().substring(trs.get(10).text().lastIndexOf("联系人："), trs.get(10).text().lastIndexOf("联系方式")).replace("联系人：", ""));
//					System.out.println("联系方式=" + ps.get(13).text().substring(ps.get(13).text().lastIndexOf("联系方式：")));
					crawling.setAgencyContactsTel(trs.get(10).text().substring(trs.get(10).text().lastIndexOf("联系方式：")).replace("联系方式：", ""));
				}
				
				
				//获取附件地址
				Element file_list = doc.getElementById("file_list");
				if(null != file_list) {
					Elements files = file_list.getElementsByTag("a");
					if(null != files && files.size() > 0) {
						String href = files.get(0).attr("href");
						crawling.setAnnex(href);
					}
				}
				
				crmCrawlingService.insertCrmCrawling(crawling);//保存数据
				
			}else {//省市县意向
				conTable = ps.get(0).getElementsByTag("div").get(0).getElementsByTag("div").get(1).getElementsByTag("tr");
				if(conTable.size() > 1) {
					for(int i = 1; i < conTable.size(); i++) {
						Elements tdCons = conTable.get(i).getElementsByTag("td");
						crawling.setDemandName(tdCons.get(1).text());//项目名称
						crawling.setSummary(tdCons.get(2).text());//项目概述
						crawling.setDemandBudget(tdCons.get(3).text());//项目预算
						crawling.setEstimateTime(formatter1.parse(tdCons.get(4).text()));//预计采购时间
						
						//获取采购单位
						crawling.setBuyingUnit(contents.get(0).getElementsByTag("font").get(0).text());
						
						crmCrawlingService.insertCrmCrawling(crawling);//保存数据
					}
				}
			}
		}
		
	}
	
	/**
	 * 生需求数据拼装+数据入库-需求
	 * @param page
	 * @throws ParseException 
	 */
	@Log(title = "数据处理-需求", businessType = BusinessType.INSERT)
	public void assemblyDataxq(Page page, String type) throws ParseException {
		
		CrmCrawling crawling = new CrmCrawling();
		
		//原文地址
		crawling.setUrl(page.url());
		//类型
		crawling.setType(type);
		
		SimpleDateFormat formatter=new SimpleDateFormat("yyyy年MM月dd日 HH时mm分ss秒");
		SimpleDateFormat formatter1=new SimpleDateFormat("yyyy年MM月dd日");
		
		Document doc = page.doc();
		//获取需求名称
		Elements names = doc.getElementsByTag("h1");
		if(names.size() > 0) {
//			System.out.println("需求名称=" + names.get(0).text());
			crawling.setDemandName(names.get(0).text());
		}
		//获取发布时间、发布人
		Elements fbsj_fbr = doc.getElementsByTag("midea");
		if(fbsj_fbr.size() > 1) {
//			System.out.println("发布时间=" + fbsj_fbr.get(0).text().replace("发布时间：", ""));
			crawling.setPublishTime(formatter.parse(fbsj_fbr.get(0).text().replace("发布时间：", "")));
//			System.out.println("发布人=" + fbsj_fbr.get(1).text().replace("发布人：", ""));
			crawling.setPublisher(fbsj_fbr.get(1).text().replace("发布人：", ""));
		}
		
		Elements contents = doc.getElementsByClass("content");//文章主体
		crawling.setDemandContent(contents.html());
		Elements ps = null;//正文内容
		if(contents.size() > 0) {
			ps = contents.get(0).getElementsByTag("p");
		}
		
		//获取概述、预算
		if(null != ps && ps.size() > 1) {
//			System.out.println("概述、预算=" + ps.get(1).text());
			crawling.setSummary(ps.get(1).text());
		}
		
		//获取公示时间
		if(null != ps && ps.size() > 6) {
//			System.out.println("开始时间=" + ps.get(6).text().substring(ps.get(6).text().lastIndexOf("：自"), ps.get(6).text().lastIndexOf("起，")).replace("：自", ""));
			crawling.setStarttime(formatter1.parse(ps.get(6).text().substring(ps.get(6).text().lastIndexOf("：自"), ps.get(6).text().lastIndexOf("起，")).replace("：自", "")));
//			System.out.println("结束时间=" + ps.get(6).text().substring(ps.get(6).text().lastIndexOf("，至"), ps.get(6).text().lastIndexOf("止")).replace("，至", ""));
			crawling.setEndtime(formatter1.parse(ps.get(6).text().substring(ps.get(6).text().lastIndexOf("，至"), ps.get(6).text().lastIndexOf("止")).replace("，至", "")));
		}
		
		//获取联系方式（采购单位、代理单位）
		if(null != ps && ps.size() > 10) {
//			System.out.println("采购单位=" + ps.get(10).text().substring(ps.get(10).text().lastIndexOf("采购单位："), ps.get(10).text().lastIndexOf("地址：")).replace("采购单位：", ""));
			crawling.setBuyingUnit(ps.get(10).text().substring(ps.get(10).text().lastIndexOf("采购单位："), ps.get(10).text().lastIndexOf("地址：")).replace("采购单位：", ""));
//			System.out.println("地址=" + ps.get(10).text().substring(ps.get(10).text().lastIndexOf("地址：")));
			crawling.setAddress(ps.get(10).text().substring(ps.get(10).text().lastIndexOf("地址：")).replace("地址：", ""));
		}
		if(null != ps && ps.size() > 11) {
//			System.out.println("联系人=" + ps.get(11).text().substring(ps.get(11).text().lastIndexOf("联系人："), ps.get(11).text().lastIndexOf("联系方式")).replace("联系人：", ""));
			crawling.setContacts(ps.get(11).text().substring(ps.get(11).text().lastIndexOf("联系人："), ps.get(11).text().lastIndexOf("联系方式")).replace("联系人：", ""));
//			System.out.println("联系方式=" + ps.get(11).text().substring(ps.get(11).text().lastIndexOf("联系方式：")));
			crawling.setContactsTel(ps.get(11).text().substring(ps.get(11).text().lastIndexOf("联系方式：")).replace("联系方式：", ""));
		}
		if(null != ps && ps.size() > 12) {
//			System.out.println("采购代理机构=" + ps.get(12).text().substring(ps.get(12).text().lastIndexOf("采购代理机构："), ps.get(12).text().lastIndexOf("地址")).replace("采购代理机构：", ""));
			crawling.setAgency(ps.get(12).text().substring(ps.get(12).text().lastIndexOf("采购代理机构："), ps.get(12).text().lastIndexOf("地址")).replace("采购代理机构：", ""));
//			System.out.println("地址=" + ps.get(12).text().substring(ps.get(12).text().lastIndexOf("地址：")));
			crawling.setAgencyAddress(ps.get(12).text().substring(ps.get(12).text().lastIndexOf("地址：")).replace("地址：", ""));
		}
		if(null != ps && ps.size() > 13) {
//			System.out.println("联系人=" + ps.get(13).text().substring(ps.get(13).text().lastIndexOf("联系人："), ps.get(13).text().lastIndexOf("联系方式")).replace("联系人：", ""));
			crawling.setAgencyContacts(ps.get(13).text().substring(ps.get(13).text().lastIndexOf("联系人："), ps.get(13).text().lastIndexOf("联系方式")).replace("联系人：", ""));
//			System.out.println("联系方式=" + ps.get(13).text().substring(ps.get(13).text().lastIndexOf("联系方式：")));
			crawling.setAgencyContactsTel(ps.get(13).text().substring(ps.get(13).text().lastIndexOf("联系方式：")).replace("联系方式：", ""));
		}
		
		
		//获取附件地址
		Element file_list = doc.getElementById("file_list");
		if(null != file_list) {
			Elements files = file_list.getElementsByTag("a");
			if(null != files && files.size() > 0) {
				String href = files.get(0).attr("href");
				String id = href.replace("javascript:openFile(", "").replace(")", "");
//				System.out.println("附件地址=" + "http://www.ccgp-shandong.gov.cn/sdgp2017/servlet/attach?id=" + id);
				crawling.setAnnex("http://www.ccgp-shandong.gov.cn/sdgp2017/servlet/attach?id=" + id);
			}
		}
		
		crmCrawlingService.insertCrmCrawling(crawling);
	}
	
	

	public static void main(String[] args) throws Exception {
		/**
		 * DemoAutoNewsCrawler 构造器中会进行 数据初始化，这两个参数接着会传给父类 super(crawlPath, autoParse);
		 * crawlPath：表示设置保存爬取记录的文件夹，本例运行之后会在应用根目录下生成一个 "crawl" 目录存放爬取信息
		 */
		
		DemoAutoNewsCrawler crawler = new DemoAutoNewsCrawler("crawl", true, null);
		
		/**
		 * 启动爬虫，爬取的深度为4层 添加的第一层种子链接,为第1层
		 */
		crawler.start(50);
	}

}